{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import tarfile\n",
    "\n",
    "import requests\n",
    "\n",
    "from pugnlp.futil import path_status, find_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# From the nlpia package for downloading data too big for the repo\n",
    "\n",
    "BIG_URLS = {\n",
    "    'w2v': (\n",
    "        'https://www.dropbox.com/s/965dir4dje0hfi4/GoogleNews-vectors-negative300.bin.gz?dl=1',\n",
    "        1647046227,\n",
    "    ),\n",
    "    'slang': (\n",
    "        'https://www.dropbox.com/s/43c22018fbfzypd/slang.csv.gz?dl=1',\n",
    "        117633024,\n",
    "    ),\n",
    "    'tweets': (\n",
    "        'https://www.dropbox.com/s/5gpb43c494mc8p0/tweets.csv.gz?dl=1',\n",
    "        311725313,\n",
    "    ),\n",
    "    'lsa_tweets': (\n",
    "        'https://www.dropbox.com/s/rpjt0d060t4n1mr/lsa_tweets_5589798_2003588x200.tar.gz?dl=1',\n",
    "        3112841563,  # 3112841312,\n",
    "    ),\n",
    "    'imdb': (\n",
    "        'https://www.dropbox.com/s/yviic64qv84x73j/aclImdb_v1.tar.gz?dl=1',\n",
    "        3112841563,  # 3112841312,\n",
    "    ),\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# These functions are part of the nlpia package which can be pip installed and run from there.\n",
    "def dropbox_basename(url):\n",
    "    filename = os.path.basename(url)\n",
    "    match = re.findall(r'\\?dl=[0-9]$', filename)\n",
    "    if match:\n",
    "        return filename[:-len(match[0])]\n",
    "    return filename\n",
    "\n",
    "def download_file(url, data_path='.', filename=None, size=None, chunk_size=4096, verbose=True):\n",
    "    \"\"\"Uses stream=True and a reasonable chunk size to be able to download large (GB) files over https\"\"\"\n",
    "    if filename is None:\n",
    "        filename = dropbox_basename(url)\n",
    "    file_path = os.path.join(data_path, filename)\n",
    "    if url.endswith('?dl=0'):\n",
    "        url = url[:-1] + '1'  # noninteractive download\n",
    "    if verbose:\n",
    "        tqdm_prog = tqdm\n",
    "        print('requesting URL: {}'.format(url))\n",
    "    else:\n",
    "        tqdm_prog = no_tqdm\n",
    "    r = requests.get(url, stream=True, allow_redirects=True)\n",
    "    size = r.headers.get('Content-Length', None) if size is None else size\n",
    "    print('remote size: {}'.format(size))\n",
    "\n",
    "    stat = path_status(file_path)\n",
    "    print('local size: {}'.format(stat.get('size', None)))\n",
    "    if stat['type'] == 'file' and stat['size'] == size:  # TODO: check md5 or get the right size of remote file\n",
    "        r.close()\n",
    "        return file_path\n",
    "\n",
    "    print('Downloading to {}'.format(file_path))\n",
    "\n",
    "    with open(file_path, 'wb') as f:\n",
    "        for chunk in r.iter_content(chunk_size=chunk_size):\n",
    "            if chunk:  # filter out keep-alive chunks\n",
    "                f.write(chunk)\n",
    "\n",
    "    r.close()\n",
    "    return file_path\n",
    "\n",
    "def untar(fname):\n",
    "    if fname.endswith(\"tar.gz\"):\n",
    "        with tarfile.open(fname) as tf:\n",
    "            tf.extractall()\n",
    "    else:\n",
    "        print(\"Not a tar.gz file: {}\".format(fname))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#  UNCOMMENT these 2 lines if you haven't already download the word2vec model and the imdb dataset\n",
    "# download_file(BIG_URLS['w2v'][0])\n",
    "# untar(download_file(BIG_URLS['imdb'][0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import glob\n",
    "import os\n",
    "\n",
    "from random import shuffle\n",
    "\n",
    "def pre_process_data(filepath):\n",
    "    \"\"\"\n",
    "    This is dependent on your training data source but we will try to generalize it as best as possible.\n",
    "    \"\"\"\n",
    "    positive_path = os.path.join(filepath, 'pos')\n",
    "    negative_path = os.path.join(filepath, 'neg')\n",
    "    \n",
    "    pos_label = 1\n",
    "    neg_label = 0\n",
    "    \n",
    "    dataset = []\n",
    "    \n",
    "    for filename in glob.glob(os.path.join(positive_path, '*.txt')):\n",
    "        with open(filename, 'r') as f:\n",
    "            dataset.append((pos_label, f.read()))\n",
    "            \n",
    "    for filename in glob.glob(os.path.join(negative_path, '*.txt')):\n",
    "        with open(filename, 'r') as f:\n",
    "            dataset.append((neg_label, f.read()))\n",
    "    \n",
    "    shuffle(dataset)\n",
    "    \n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using Theano backend.\n"
     ]
    }
   ],
   "source": [
    "from nltk.tokenize import TreebankWordTokenizer\n",
    "from gensim.models import KeyedVectors\n",
    "word_vectors = KeyedVectors.load_word2vec_format('GoogleNews-vectors-negative300.bin.gz', binary=True, limit=200000)\n",
    "\n",
    "def tokenize_and_vectorize(dataset):\n",
    "    tokenizer = TreebankWordTokenizer()\n",
    "    vectorized_data = []\n",
    "    expected = []\n",
    "    for sample in dataset:\n",
    "        tokens = tokenizer.tokenize(sample[1])\n",
    "        sample_vecs = []\n",
    "        for token in tokens:\n",
    "            try:\n",
    "                sample_vecs.append(word_vectors[token])\n",
    "\n",
    "            except KeyError:\n",
    "                pass  # No matching token in the Google w2v vocab\n",
    "            \n",
    "        vectorized_data.append(sample_vecs)\n",
    "\n",
    "    return vectorized_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  5.12695312e-02,  -2.23388672e-02,  -1.72851562e-01,\n",
       "         1.61132812e-01,  -8.44726562e-02,   5.73730469e-02,\n",
       "         5.85937500e-02,  -8.25195312e-02,  -1.53808594e-02,\n",
       "        -6.34765625e-02,   1.79687500e-01,  -4.23828125e-01,\n",
       "        -2.25830078e-02,  -1.66015625e-01,  -2.51464844e-02,\n",
       "         1.07421875e-01,  -1.99218750e-01,   1.59179688e-01,\n",
       "        -1.87500000e-01,  -1.20117188e-01,   1.55273438e-01,\n",
       "        -9.91210938e-02,   1.42578125e-01,  -1.64062500e-01,\n",
       "        -8.93554688e-02,   2.00195312e-01,  -1.49414062e-01,\n",
       "         3.20312500e-01,   3.28125000e-01,   2.44140625e-02,\n",
       "        -9.71679688e-02,  -8.20312500e-02,  -3.63769531e-02,\n",
       "        -8.59375000e-02,  -9.86328125e-02,   7.78198242e-03,\n",
       "        -1.34277344e-02,   5.27343750e-02,   1.48437500e-01,\n",
       "         3.33984375e-01,   1.66015625e-02,  -2.12890625e-01,\n",
       "        -1.50756836e-02,   5.24902344e-02,  -1.07421875e-01,\n",
       "        -8.88671875e-02,   2.49023438e-01,  -7.03125000e-02,\n",
       "        -1.59912109e-02,   7.56835938e-02,  -7.03125000e-02,\n",
       "         1.19140625e-01,   2.29492188e-01,   1.41601562e-02,\n",
       "         1.15234375e-01,   7.50732422e-03,   2.75390625e-01,\n",
       "        -2.44140625e-01,   2.96875000e-01,   3.49121094e-02,\n",
       "         2.42187500e-01,   1.35742188e-01,   1.42578125e-01,\n",
       "         1.75781250e-02,   2.92968750e-02,  -1.21582031e-01,\n",
       "         2.28271484e-02,  -4.76074219e-02,  -1.55273438e-01,\n",
       "         3.14331055e-03,   3.45703125e-01,   1.22558594e-01,\n",
       "        -1.95312500e-01,   8.10546875e-02,  -6.83593750e-02,\n",
       "        -1.47094727e-02,   2.14843750e-01,  -1.21093750e-01,\n",
       "         1.57226562e-01,  -2.07031250e-01,   1.36718750e-01,\n",
       "        -1.29882812e-01,   5.29785156e-02,  -2.71484375e-01,\n",
       "        -2.98828125e-01,  -1.84570312e-01,  -2.29492188e-01,\n",
       "         1.19140625e-01,   1.53198242e-02,  -2.61718750e-01,\n",
       "        -1.23046875e-01,  -1.86767578e-02,  -6.49414062e-02,\n",
       "        -8.15429688e-02,   7.86132812e-02,  -3.53515625e-01,\n",
       "         5.24902344e-02,  -2.45361328e-02,  -5.43212891e-03,\n",
       "        -2.08984375e-01,  -2.10937500e-01,  -1.79687500e-01,\n",
       "         2.42187500e-01,   2.57812500e-01,   1.37695312e-01,\n",
       "        -2.10937500e-01,  -2.17285156e-02,  -1.38671875e-01,\n",
       "         1.84326172e-02,  -1.23901367e-02,  -1.59179688e-01,\n",
       "         1.61132812e-01,   2.08007812e-01,   1.03027344e-01,\n",
       "         9.81445312e-02,  -6.83593750e-02,  -8.72802734e-03,\n",
       "        -2.89062500e-01,  -2.14843750e-01,  -1.14257812e-01,\n",
       "        -2.21679688e-01,   4.12597656e-02,  -3.12500000e-01,\n",
       "        -5.59082031e-02,  -9.76562500e-02,   5.81054688e-02,\n",
       "        -4.05273438e-02,  -1.73828125e-01,   1.64062500e-01,\n",
       "        -2.53906250e-01,  -1.54296875e-01,  -2.31933594e-02,\n",
       "        -2.38281250e-01,   2.07519531e-02,  -2.73437500e-01,\n",
       "         3.90625000e-03,   1.13769531e-01,  -1.73828125e-01,\n",
       "         2.57812500e-01,   2.35351562e-01,   5.22460938e-02,\n",
       "         6.83593750e-02,  -1.75781250e-01,   1.60156250e-01,\n",
       "        -5.98907471e-04,   5.98144531e-02,  -2.11914062e-01,\n",
       "        -5.54199219e-02,  -7.51953125e-02,  -3.06640625e-01,\n",
       "         4.27734375e-01,   5.32226562e-02,  -2.08984375e-01,\n",
       "        -5.71289062e-02,  -2.09960938e-01,   3.29589844e-02,\n",
       "         1.05468750e-01,  -1.50390625e-01,  -9.37500000e-02,\n",
       "         1.16699219e-01,   6.44531250e-02,   2.80761719e-02,\n",
       "         2.41210938e-01,  -1.25976562e-01,  -1.00585938e-01,\n",
       "        -1.22680664e-02,  -3.26156616e-04,   1.58691406e-02,\n",
       "         1.27929688e-01,  -3.32031250e-02,   4.07714844e-02,\n",
       "        -1.31835938e-01,   9.81445312e-02,   1.74804688e-01,\n",
       "        -2.36328125e-01,   5.17578125e-02,   1.83593750e-01,\n",
       "         2.42919922e-02,  -4.31640625e-01,   2.46093750e-01,\n",
       "        -3.03955078e-02,  -2.47802734e-02,  -1.17187500e-01,\n",
       "         1.61132812e-01,  -5.71289062e-02,   1.16577148e-02,\n",
       "         2.81250000e-01,   4.27734375e-01,   4.56542969e-02,\n",
       "         1.01074219e-01,  -3.95507812e-02,   1.77001953e-02,\n",
       "        -8.98437500e-02,   1.35742188e-01,   2.08007812e-01,\n",
       "         1.88476562e-01,  -1.52343750e-01,  -2.37304688e-01,\n",
       "        -1.90429688e-01,   7.12890625e-02,  -2.46093750e-01,\n",
       "        -2.61718750e-01,  -2.34375000e-01,  -1.45507812e-01,\n",
       "        -1.17187500e-02,  -1.50390625e-01,  -1.13281250e-01,\n",
       "         1.82617188e-01,   2.63671875e-01,  -1.37695312e-01,\n",
       "        -4.58984375e-01,  -4.68750000e-02,  -1.26953125e-01,\n",
       "        -4.22363281e-02,  -1.66992188e-01,   1.26953125e-01,\n",
       "         2.59765625e-01,  -2.44140625e-01,  -2.19726562e-01,\n",
       "        -8.69140625e-02,   1.59179688e-01,  -3.78417969e-02,\n",
       "         8.97216797e-03,  -2.77343750e-01,  -1.04980469e-01,\n",
       "        -1.75781250e-01,   2.28515625e-01,  -2.70996094e-02,\n",
       "         2.85156250e-01,  -2.73437500e-01,   1.61132812e-02,\n",
       "         5.90820312e-02,  -2.39257812e-01,   1.77734375e-01,\n",
       "        -1.34765625e-01,   1.38671875e-01,   3.53515625e-01,\n",
       "         1.22070312e-01,   1.43554688e-01,   9.22851562e-02,\n",
       "         2.29492188e-01,  -3.00781250e-01,  -4.88281250e-02,\n",
       "        -1.79687500e-01,   2.96875000e-01,   1.75781250e-01,\n",
       "         4.80957031e-02,  -3.38745117e-03,   7.91015625e-02,\n",
       "        -2.38281250e-01,  -2.31445312e-01,   1.66015625e-01,\n",
       "        -2.13867188e-01,  -7.03125000e-02,  -7.56835938e-02,\n",
       "         1.96289062e-01,  -1.29882812e-01,  -1.05957031e-01,\n",
       "        -3.53515625e-01,  -1.16699219e-01,  -5.10253906e-02,\n",
       "         3.39355469e-02,  -1.43554688e-01,  -3.90625000e-03,\n",
       "         1.73828125e-01,  -9.96093750e-02,  -1.66015625e-01,\n",
       "        -8.54492188e-02,  -3.82812500e-01,   5.90820312e-02,\n",
       "        -6.22558594e-02,   8.83789062e-02,  -8.88671875e-02,\n",
       "         3.28125000e-01,   6.83593750e-02,  -1.91406250e-01,\n",
       "        -8.35418701e-04,   1.04003906e-01,   1.52343750e-01,\n",
       "        -1.53350830e-03,   4.16015625e-01,  -3.32031250e-02,\n",
       "         1.49414062e-01,   2.42187500e-01,  -1.76757812e-01,\n",
       "        -4.93164062e-02,  -1.24511719e-01,   1.25976562e-01,\n",
       "         1.74804688e-01,   2.81250000e-01,  -1.80664062e-01,\n",
       "         1.03027344e-01,  -2.75390625e-01,   2.61718750e-01,\n",
       "         2.46093750e-01,  -4.71191406e-02,   6.25000000e-02,\n",
       "         4.16015625e-01,  -3.55468750e-01,   2.22656250e-01], dtype=float32)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_vectors[\"dog\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def collect_expected(dataset):\n",
    "    \"\"\" Peel of the target values from the dataset \"\"\"\n",
    "    expected = []\n",
    "    for sample in dataset:\n",
    "        expected.append(sample[0])\n",
    "    return expected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dataset = pre_process_data('./aclImdb_v1/train')\n",
    "vectorized_data = tokenize_and_vectorize(dataset)\n",
    "expected = collect_expected(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "split_point = int(len(vectorized_data)*.8)\n",
    "\n",
    "x_train = vectorized_data[:split_point]\n",
    "y_train = expected[:split_point]\n",
    "x_test = vectorized_data[split_point:]\n",
    "y_test = expected[split_point:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "maxlen = 400\n",
    "batch_size = 32         # How many samples to show the net before backpropogating the error and updating the weights\n",
    "embedding_dims = 300    # Length of the token vectors we will create for passing into the Convnet\n",
    "\n",
    "epochs = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def pad_trunc(data, maxlen):\n",
    "    \"\"\" For a given dataset pad with zero vectors or truncate to maxlen \"\"\"\n",
    "    new_data = []\n",
    "\n",
    "    # Create a vector of 0's the length of our word vectors\n",
    "    zero_vector = []\n",
    "    for _ in range(len(data[0][0])):\n",
    "        zero_vector.append(0.0)\n",
    "\n",
    "    for sample in data:\n",
    " \n",
    "        if len(sample) > maxlen:\n",
    "            temp = sample[:maxlen]\n",
    "        elif len(sample) < maxlen:\n",
    "            temp = sample\n",
    "            additional_elems = maxlen - len(sample)\n",
    "            for _ in range(additional_elems):\n",
    "                temp.append(zero_vector)\n",
    "        else:\n",
    "            temp = sample\n",
    "        new_data.append(temp)\n",
    "    return new_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "x_train = pad_trunc(x_train, maxlen)\n",
    "x_test = pad_trunc(x_test, maxlen)\n",
    "\n",
    "x_train = np.reshape(x_train, (len(x_train), maxlen, embedding_dims))\n",
    "y_train = np.array(y_train)\n",
    "x_test = np.reshape(x_test, (len(x_test), maxlen, embedding_dims))\n",
    "y_test = np.array(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Build model...\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "simple_rnn_1 (SimpleRNN)     (None, 400, 50)           17550     \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 400, 50)           0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 20000)             0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1)                 20001     \n",
      "=================================================================\n",
      "Total params: 37,551\n",
      "Trainable params: 37,551\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Flatten, SimpleRNN\n",
    "\n",
    "num_neurons = 50\n",
    "\n",
    "print('Build model...')\n",
    "model = Sequential()\n",
    "\n",
    "model.add(SimpleRNN(num_neurons, return_sequences=True, input_shape=(maxlen, embedding_dims)))\n",
    "model.add(Dropout(.2))\n",
    "\n",
    "model.add(Flatten())\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile('rmsprop', 'binary_crossentropy',  metrics=['accuracy'])\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 20000 samples, validate on 5000 samples\n",
      "Epoch 1/2\n",
      "20000/20000 [==============================] - 337s - loss: 0.5820 - acc: 0.7046 - val_loss: 0.4991 - val_acc: 0.7824\n",
      "Epoch 2/2\n",
      "20000/20000 [==============================] - 247s - loss: 0.4261 - acc: 0.8123 - val_loss: 0.5390 - val_acc: 0.7552\n",
      "Model saved.\n"
     ]
    }
   ],
   "source": [
    "model.fit(x_train, y_train,\n",
    "          batch_size=batch_size,\n",
    "          epochs=epochs,\n",
    "          validation_data=(x_test, y_test))\n",
    "model_structure = model.to_json()\n",
    "with open(\"simplernn_model1.json\", \"w\") as json_file:\n",
    "    json_file.write(model_structure)\n",
    "\n",
    "model.save_weights(\"simplernn_weights1.h5\")\n",
    "print('Model saved.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.models import model_from_json\n",
    "with open(\"simplernn_model1.json\", \"r\") as json_file:\n",
    "    json_string = json_file.read()\n",
    "model = model_from_json(json_string)\n",
    "\n",
    "model.load_weights('simplernn_weights1.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sample_1 = \"I'm hate that the dismal weather that had me down for so long, when will it break! Ugh, when does happiness return?  The sun is blinding and the puffy clouds are too thin.  I can't wait for the weekend.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# We pass a dummy value in the first element of the tuple just because our helper expects it from the way processed the initial data.  That value won't ever see the network, so it can be whatever.\n",
    "vec_list = tokenize_and_vectorize([(1, sample_1)])\n",
    "\n",
    "# Tokenize returns a list of the data (length 1 here)\n",
    "test_vec_list = pad_trunc(vec_list, maxlen)\n",
    "\n",
    "test_vec = np.reshape(test_vec_list, (len(test_vec_list), maxlen, embedding_dims))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1 [==============================] - 0s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0]], dtype=int32)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.predict_classes(test_vec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Build model...\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "simple_rnn_2 (SimpleRNN)     (None, 400, 100)          40100     \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 400, 100)          0         \n",
      "_________________________________________________________________\n",
      "flatten_2 (Flatten)          (None, 40000)             0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 1)                 40001     \n",
      "=================================================================\n",
      "Total params: 80,101\n",
      "Trainable params: 80,101\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Flatten, SimpleRNN\n",
    "\n",
    "num_neurons = 100\n",
    "\n",
    "print('Build model...')\n",
    "model = Sequential()\n",
    "\n",
    "model.add(SimpleRNN(num_neurons, return_sequences=True, input_shape=(maxlen, embedding_dims)))\n",
    "model.add(Dropout(.2))\n",
    "\n",
    "model.add(Flatten())\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile('rmsprop', 'binary_crossentropy',  metrics=['accuracy'])\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 20000 samples, validate on 5000 samples\n",
      "Epoch 1/2\n",
      " 6464/20000 [========>.....................] - ETA: 229s - loss: 0.8151 - acc: 0.5860"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-17-75f17f7d4b4c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m           \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m           \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m           validation_data=(x_test, y_test))\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0mmodel_structure\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_json\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"simplernn_model2.json\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"w\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mjson_file\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.virtualenvs/nltk/lib/python3.6/site-packages/keras/models.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, **kwargs)\u001b[0m\n\u001b[1;32m    861\u001b[0m                               \u001b[0mclass_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mclass_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    862\u001b[0m                               \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 863\u001b[0;31m                               initial_epoch=initial_epoch)\n\u001b[0m\u001b[1;32m    864\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    865\u001b[0m     def evaluate(self, x, y, batch_size=32, verbose=1,\n",
      "\u001b[0;32m~/.virtualenvs/nltk/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, **kwargs)\u001b[0m\n\u001b[1;32m   1428\u001b[0m                               \u001b[0mval_f\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_f\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_ins\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_ins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1429\u001b[0m                               \u001b[0mcallback_metrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback_metrics\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1430\u001b[0;31m                               initial_epoch=initial_epoch)\n\u001b[0m\u001b[1;32m   1431\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1432\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.virtualenvs/nltk/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_fit_loop\u001b[0;34m(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch)\u001b[0m\n\u001b[1;32m   1077\u001b[0m                 \u001b[0mbatch_logs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'size'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1078\u001b[0m                 \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_logs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1079\u001b[0;31m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1080\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1081\u001b[0m                     \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.virtualenvs/nltk/lib/python3.6/site-packages/keras/backend/theano_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   1195\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1196\u001b[0m         \u001b[0;32massert\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1197\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1199\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.virtualenvs/nltk/lib/python3.6/site-packages/theano/compile/function_module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    882\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    883\u001b[0m             \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 884\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0moutput_subset\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    885\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput_subset\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moutput_subset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    886\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.virtualenvs/nltk/lib/python3.6/site-packages/theano/scan_module/scan_op.py\u001b[0m in \u001b[0;36mrval\u001b[0;34m(p, i, o, n, allow_gc)\u001b[0m\n\u001b[1;32m    987\u001b[0m         def rval(p=p, i=node_input_storage, o=node_output_storage, n=node,\n\u001b[1;32m    988\u001b[0m                  allow_gc=allow_gc):\n\u001b[0;32m--> 989\u001b[0;31m             \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mo\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    990\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    991\u001b[0m                 \u001b[0mcompute_map\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.virtualenvs/nltk/lib/python3.6/site-packages/theano/scan_module/scan_op.py\u001b[0m in \u001b[0;36mp\u001b[0;34m(node, args, outs)\u001b[0m\n\u001b[1;32m    976\u001b[0m                                                 \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    977\u001b[0m                                                 \u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 978\u001b[0;31m                                                 self, node)\n\u001b[0m\u001b[1;32m    979\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mImportError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtheano\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgof\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcmodule\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mMissingGXX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    980\u001b[0m             \u001b[0mp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model.fit(x_train, y_train,\n",
    "          batch_size=batch_size,\n",
    "          epochs=epochs,\n",
    "          validation_data=(x_test, y_test))\n",
    "model_structure = model.to_json()\n",
    "with open(\"simplernn_model2.json\", \"w\") as json_file:\n",
    "    json_file.write(model_structure)\n",
    "\n",
    "model.save_weights(\"simplernn_weights2.h5\")\n",
    "print('Model saved.')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
